package com.example.book.crawler.pageprocessor;

import com.example.book.crawler.entity.CrawlerContent;
import com.example.book.in.entity.BookInfo;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

public class CategoryPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
            .setDomain("www.baidu.com");

    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void process(Page page) {
        List<BookInfo> list = new ArrayList<>();
        BookInfo bookInfo = null;
        boolean a = true;
        int i = 1;
        while (a) {

            String name = page.getHtml().xpath("/html/body/div[2]/div[1]/div[2]/div/div[" + i + "]/div/div[2]/div/h4/a/text()").toString();
            String href = page.getHtml().xpath("/html/body/div[2]/div[1]/div[2]/div/div[" + i + "]/div/div[2]/div/h4/a/@href").toString();
            String img = page.getHtml().xpath("/html/body/div[2]/div[1]/div[2]/div/div[" + i + "]/div/div[1]/a/img/@src").toString();
            String author = page.getHtml().xpath("/html/body/div[2]/div[1]/div[2]/div/div[" + i + "]/div/div[2]/div/small/text()").toString();
            String desc = page.getHtml().xpath("/html/body/div[2]/div[1]/div[2]/div/div[" + i + "]/div/div[2]/div/p/text()").toString();

            if (StringUtils.isBlank(name)) {
                a = false;
                continue;
            }

            bookInfo = new BookInfo();

            bookInfo.setBookName(name);
            bookInfo.setBookAuthor(author);
            bookInfo.setBookIntroduction(desc);
            bookInfo.setBookCover(img);
            bookInfo.setRemarks(href);
            bookInfo.setBookStatus("3");//新增数据
            bookInfo.setStatus("0");

            list.add(bookInfo);
            page.putField("list", list);
            i++;
        }
    }

    public static void main(String[] args) {
        new Spider(new CategoryPageProcessor())

                .setUUID(UUID.randomUUID().toString())
                .addUrl("https://www.wmtxt.net/fenlei1/1.html").addPipeline((resultItems, task) -> {
                    List<CrawlerContent> list = resultItems.get("list");
                    System.out.println(">>>>" + list.size());
                }).thread(1).run();
    }
}
